#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# @Time    : 2018/12/3 21:36
# @Author  : liujiantao
# @Site    : 
# @File    : feature_select.py
# @Software: PyCharm
from tiancheng.base.base_helper import *
# train, label, cols = get_X_y()
y = get_tag_train_new()[tag_hd.UID].to_frame()
sub = get_sub()[tag_hd.UID].to_frame()
train = pd.read_csv(features_base_path+"train_media_ftr.csv")
y = y.merge(train, on='UID', how='left')
op = pd.read_csv(features_base_path+"operation_feature.csv")
test_op = op[op[tag_hd.Tag]==-1]
sub = sub.merge(test_op, on='UID', how='left')
train_op = op[op[tag_hd.Tag]!=-1]
y = y.merge(train_op, on='UID', how='left')
tst = pd.read_csv(features_base_path+"transction_feature.csv")
test_tst = tst[tst[tag_hd.Tag]==-1]
sub = sub.merge(test_tst, on='UID', how='left')
train_tst = tst[tst[tag_hd.Tag]!=-1]
train = y.merge(train_tst, on='UID', how='left')
test = pd.read_csv(features_base_path+"test_media_ftr.csv")
test = sub.merge(test, on='UID', how='left')
test = fill_mean(test).fillna(-1)
print(test.shape)
test_data = VarianceThreshold_selector(test,threshold=1)
test_cols = list(test_data.columns.values)
# tr_corr
train = fill_mean(train).fillna(-1)
print(train.shape)
tr_corr = train.corr()[tag_hd.Tag].reset_index()
tr_corr = tr_corr[['Tag','index']].fillna(0).sort_values(by=['Tag'], ascending=False)
print(tr_corr)
col = tr_corr[(tr_corr['Tag'].abs() >= 0.02) & (tr_corr['Tag'].abs()<0.6)]['index'].values
print(col)
col =  list(set(col).intersection(set(test_cols)))
train_data = train[col]
test_data = test[col]
print(train_data.shape)

# test_data.pop(tag_hd.Tag)
# test_data.head(2)
# # test_data.pop(tag_hd.Tag)
# train_data.head(2)
train_data.to_csv(features_base_path + 'train_select_ftr.csv', index=False)
test_data.to_csv(features_base_path + 'test_select_ftr.csv', index=False)
print(train_data.shape)
print(test_data.shape)
print(123)